
#from tqdm import tqdm
import time
import pandas as pd
import scrapy
from scrapy.http import Request

#import web


from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all" 


# 允许的站点域
allow_domain = "localbtcoin.com"


ads_data = []
phone_data = []
res_urls = []
    
def is_number(s):
        try:
            int(s)
            return True
        except ValueError:
            pass
 
        try:
            import unicodedata
            unicodedata.numeric(s)
            return True
        except (TypeError, ValueError):
            pass
 
        return False

def get_phone(strphone):
        #get phone number from ads
        adslen = len(strphone)
        getphone = '' 
        for i in (range(adslen)):
            ch = strphone[i:i+1]
            if  (ch == '1' and adslen-i>10):
                strRightString = strphone[i:11+i]
                if (is_number(strRightString)):
                    getphone = strRightString
                break
            else:
                continue
        return getphone


def save_csv():
        table=pd.DataFrame({'urls':res_urls,'ads':ads_data,'phone':phone_data})
        date = time.strftime('%Y%m%d',time.localtime(time.time()))
        table.to_csv(r'data/localbtcoin%s.csv' % (date),encoding = "utf_8_sig")
        return


class localSpider(scrapy.Spider):
    name = "localbtcoin"
    
    headers = {
            'User-Agent': 'Mozilla/5.0'
            }   
    
    start_urls = [
        "https://localbitcoins.com/forums/"
    ]
    
    def parse(self, response):
        trade_urls = ['https://localbitcoins.com/ad/%d' % (100000+i) for i in range(500000)]
        for cate_url in trade_urls:
            yield Request(cate_url, callback=self.parse_page)

    def parse_page(self, response):
        #返回留言中的数组
        content = response.selector.xpath('//p[@class="messagetext"]/text()').extract()  
        #取到的数据连接成字符串
        detail_ads = ','.join(content)
             
        if (len(detail_ads)>0):
            ads_data.append(detail_ads)
            myphone = get_phone(detail_ads)
            phone_data.append(myphone)
            res_urls.append(response.url)
            save_csv()
        
        
           
        
    	


